package com.push.spider.spider;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.HanLP;
import com.push.common.config.MinioConfig;
import com.push.common.config.TransConfig;
import com.push.common.manager.BeanManager;
import com.push.common.utils.*;
import com.push.common.utils.httpclient.HttpClientResult;
import com.push.common.utils.httpclient.HttpClientUtil;
import com.push.spider.utils.UrlUtils;
import com.push.web.domain.BizArticle;
import com.push.web.domain.BizArticleAppendix;
import com.push.web.domain.BizLabelSecond;
import com.push.web.domain.BizSource;
import com.push.web.service.IBizArticleAppendixService;
import com.push.web.service.IBizArticleService;
import com.push.web.service.IBizLabelSecondService;
import com.push.web.service.IBizSourceService;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.*;

public class HuanqiuSpider {
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private ArrayList<String> hasDownUrls = new ArrayList<>();

    private Queue<String> queue = new LinkedList<String>();

    private BizSource source;

    private List<BizLabelSecond> labelSeconds = new ArrayList<>();

    @Autowired
    IBizArticleService articleService = BeanManager.getBean(IBizArticleService.class);

    @Autowired
    IBizSourceService sourceService = BeanManager.getBean(IBizSourceService.class);

    @Autowired
    IBizLabelSecondService labelSecondService = BeanManager.getBean(IBizLabelSecondService.class);

    @Autowired
    IBizArticleAppendixService articleAppendixService = BeanManager.getBean(IBizArticleAppendixService.class);

    @Autowired
    MinioUtil minioUtil = BeanManager.getBean(MinioUtil.class);

    @Autowired
    MinioConfig prop = BeanManager.getBean(MinioConfig.class);

    @Autowired
    private TransConfig transConfig = BeanManager.getBean(TransConfig.class);

    public int size = 0;

    public HuanqiuSpider(String sourceId) {
        this.source = sourceService.selectBizSourceBySourceId(Long.valueOf(sourceId));
        this.labelSeconds = labelSecondService.selectBizLabelSecondListByParentIds(this.source.getLabelIds());
    }

    @SneakyThrows
    public void run(){
        HashMap params = new HashMap();
        params.put("node","\"/e3pmh1dm8/e3pmt7hva\",\"/e3pmh1dm8/e3pmtdr2r\",\"/e3pmh1dm8/e3pn62l96\",\"/e3pmh1dm8/e3pn6f3oh\"");
        params.put("offset","0");
        params.put("limit","20");
        HttpClientResult clientResult = HttpClientUtil.doGet("https://mil.huanqiu.com/api/list", params,true);
        JSONObject resultObj = JSONObject.parseObject(clientResult.getContent());
        JSONArray jsonArray = resultObj.getJSONArray("list");
        for (int i = 0; i < jsonArray.size(); i++){
            JSONObject articleObj =jsonArray.getJSONObject(i);
            if (!articleObj.containsKey("title")){
                continue;
            }
            String aid = articleObj.getString("aid");
            String title = articleObj.getString("title");
            String summary = articleObj.getString("summary");
            String host = articleObj.getString("host");
            String publishTime = DateUtils.longToString(Long.valueOf(articleObj.getString("xtime")),"yyyy-MM-dd HH:mm");
            String link = articleObj.getJSONObject("source").getString("url");
            String sourceName = articleObj.getJSONObject("source").getString("name");
            String sourceUrl = "https://"+host+"/article/"+aid;
            String avator = "";
            queue.offer(sourceUrl);
            String dataKey = EncryptUtil.encrypt(sourceUrl, "md5");
            BizArticle article = buildArticle(dataKey, title, summary, sourceUrl, link, sourceName, avator, publishTime, source.getSourceId());
            BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
            if (StringUtils.isNull(bizArticle)){
                if (strContains(article.getTitleOrg(), labelSeconds)){
                    //自动翻译标题
                    if (StringUtils.isContainsChinese(article.getTitleOrg())){
                        article.setTitleCn(article.getTitleOrg());
                    } else {
                        String transWord = getTransResult(article.getTitleOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                            article.setTitleCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }

                    //自动翻译摘要
                    if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                        String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                            article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }
                    articleService.insertBizArticle(article);
                }
            } else {
                article.setId(bizArticle.getId());
                articleService.updateBizArticle(article);
            }
        }
        do{
            String url = queue.poll();
            HttpClientResult personResult = new HttpClientResult();
            if (url.startsWith("https")){
                personResult = HttpClientUtil.doGet(url, true);
            } else {
                personResult = HttpClientUtil.doGet(url);
            }
            size++;
            Html html = new Html(personResult.getContent());
            String contentsOrg = extractInfo(source.getContentsXpath(), html);
            if (source.getPicDownload().equalsIgnoreCase("是")){
                List<String> contentImgUrls = UrlUtils.praseUrlFromString(contentsOrg, url);
                if (contentImgUrls.size() > 0){
                    List<String> newImages = UrlUtils.downloadFilesToMinio(contentImgUrls, minioUtil);
                    contentsOrg = UrlUtils.repairDownloadContent(contentsOrg, newImages);
                }
            } else {
                contentsOrg = UrlUtils.repairContent(contentsOrg, url);
            }
            String dataKey = EncryptUtil.encrypt(url, "md5");
            BizArticle article = updateArticle(dataKey, contentsOrg, source.getSourceId());
            BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
            if (StringUtils.isNotNull(bizArticle)){
                article.setId(bizArticle.getId());
                //自动翻译摘要
                if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                    String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                    JSONObject titleObj = JSONObject.parseObject(transWord);
                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                    if (titleArray.size() > 0){
                        article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                    }
                }/* else if (StringUtils.isNotEmpty(article.getContentsCn())){
                    List<String> sentenceList = HanLP.extractSummary(article.getContentsCn(),10);
                    if(sentenceList.size() > 0){
                        String abstact = sentenceList.toString();
                        article.setSummaryOrg(abstact.substring(1,abstact.lastIndexOf("]")));
                        String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (titleArray.size() > 0){
                            article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }
                }*/
                articleService.updateBizArticle(article);
            }
            Thread.sleep(source.getIntervalNum()*1000);
        } while (queue.size() > 0);
    }

    public String extractInfo(String xpathString, Selectable contentNode){
        if (StringUtils.isEmpty(xpathString)){
            return "";
        }
        List<String> xpathList = Arrays.asList(xpathString.split(";"));
        for (String xpath: xpathList){
            List<Selectable> nodes = contentNode.xpath(xpath).nodes();
            if(nodes.size() > 0){
                return nodes.get(0).toString();
            }
        }
        return "";
    }
    public BizArticle buildArticle(String dataKey, String titleOrg, String summary, String url,String link, String sourceName, String avator, String publishTime, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setTitleOrg(titleOrg);
        article.setSummaryOrg(summary);

        article.setPublisher(sourceName);
        if (StringUtils.isNotEmpty(link)){
            article.setUrl(link);
        } else {
            article.setUrl(url);
        }
        article.setAvator(avator);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(publishTime);
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public BizArticle updateArticle(String dataKey, String contentsOrg, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setContentsOrg(contentsOrg);
        article.setContentsCn(StringUtils.delHtmlTags(contentsOrg));
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public static boolean strContains(String str, List<BizLabelSecond> labels){
        if (labels == null){
            return true;
        }
        if (labels.isEmpty()){
            return true;
        }
        if(StringUtils.isEmpty(str)){
            return false;
        }
        long len = labels.parallelStream().filter(label -> str.toLowerCase(Locale.ROOT).contains(label.getLabelName().toLowerCase(Locale.ROOT))).count();
        if (len > 0){
            return true;
        }
        return false;
    }

    public String getTransResult(String query, String from, String to) {
        Map<String, String> params = buildParams(query, from, to);
        return HttpClientUtil.doGet(transConfig.getTransUrl(), params,false).getContent();
    }

    private Map<String, String> buildParams(String query, String from, String to) {
        Map<String, String> params = new HashMap<String, String>();
        params.put("q", query);
        params.put("from", from);
        params.put("to", to);
        params.put("appid", transConfig.getAppid());
        // 随机数
        String salt = String.valueOf(System.currentTimeMillis());
        params.put("salt", salt);
        // 签名
        String src = transConfig.getAppid() + query + salt + transConfig.getSecurityKey();
        params.put("sign", MD5.md5(src));
        return params;
    }


    public int getSize(){
        return size;
    }
}
